# Import warnings
import warnings
warnings.filterwarnings("ignore")

# Libraries for data manipulation and visualization
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# Algorithms to use
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.model_selection import train_test_split, RandomizedSearchCV, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn import tree
from sklearn.ensemble import RandomForestClassifier
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.metrics import roc_curve, roc_auc_score, confusion_matrix, classification_report, f1_score
from sklearn import metrics
from xgboost import XGBRegressor, XGBClassifier
import multiprocessing
import shap
import xgboost as xgb

#Import csv file with data
customer = pd.read_csv("ExtraaLearn.csv")

# Copying data to another variable to avoid any changes to original data
data = customer.copy()

#View the first 5 rows of the data set
data.head()

#View the last 5 rows of the data set
data.tail()

#View info
data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 4612 entries, 0 to 4611
Data columns (total 15 columns):
 #   Column                 Non-Null Count  Dtype  
---  ------                 --------------  -----  
 0   ID                     4612 non-null   object 
 1   age                    4612 non-null   int64  
 2   current_occupation     4612 non-null   object 
 3   first_interaction      4612 non-null   object 
 4   profile_completed      4612 non-null   object 
 5   website_visits         4612 non-null   int64  
 6   time_spent_on_website  4612 non-null   int64  
 7   page_views_per_visit   4612 non-null   float64
 8   last_activity          4612 non-null   object 
 9   print_media_type1      4612 non-null   object 
 10  print_media_type2      4612 non-null   object 
 11  digital_media          4612 non-null   object 
 12  educational_channels   4612 non-null   object 
 13  referral               4612 non-null   object 
 14  status                 4612 non-null   int64  
dtypes: float64(1), int64(4), object(10)
memory usage: 540.6+ KB

data = data.drop(["ID"], axis = 1)
data.head()

#Observations
# Selecting numerical columns and checking the summary statistics
num_cols = data.select_dtypes('number').columns

data[num_cols].describe().T

#Checking the distribution and outliers for numerical columns in the data
for col in ['age', 'website_visits', 'time_spent_on_website', 'page_views_per_visit']:
    print(col)

    print('Skew :', round(data[col].skew(), 2))

    plt.figure(figsize = (15, 4))

    plt.subplot(1,2,1)

    data[col].hist(bins = 10, grid = False)

    plt.ylabel('count')

    plt.subplot(1, 2, 2)

    sns.boxplot(x = data[col])

    plt.show()

age
Skew : -0.72

website_visits
Skew : 2.16

time_spent_on_website
Skew : 0.95

page_views_per_visit
Skew : 1.27

#Check categorical variables.  Status is numerical but can also be categorical
cat_cols = ['current_occupation', 'first_interaction', 'profile_completed', 'last_activity', 'print_media_type1',
        'print_media_type2', 'digital_media', 'educational_channels', 'referral','status']

for col in cat_cols:
    print(data[col].value_counts(normalize = True))  # The parameter normalize = True gives the percentage of each category
    print('*'*40)

current_occupation
Professional    0.567216
Unemployed      0.312446
Student         0.120338
Name: proportion, dtype: float64
****************************************
first_interaction
Website       0.551171
Mobile App    0.448829
Name: proportion, dtype: float64
****************************************
profile_completed
High      0.490893
Medium    0.485906
Low       0.023200
Name: proportion, dtype: float64
****************************************
last_activity
Email Activity      0.493929
Phone Activity      0.267563
Website Activity    0.238508
Name: proportion, dtype: float64
****************************************
print_media_type1
No     0.892238
Yes    0.107762
Name: proportion, dtype: float64
****************************************
print_media_type2
No     0.94948
Yes    0.05052
Name: proportion, dtype: float64
****************************************
digital_media
No     0.885733
Yes    0.114267
Name: proportion, dtype: float64
****************************************
educational_channels
No     0.847138
Yes    0.152862
Name: proportion, dtype: float64
****************************************
referral
No     0.979835
Yes    0.020165
Name: proportion, dtype: float64
****************************************
status
0    0.701431
1    0.298569
Name: proportion, dtype: float64
****************************************

#Bivariate Analysis Overview
# List of columns to plot
columns_to_plot = ['current_occupation', 'first_interaction', 'profile_completed',
                   'last_activity', 'print_media_type1', 'print_media_type2',
                   'digital_media', 'educational_channels', 'referral']

# Loop through each column
for col in columns_to_plot:

    # Create smaller count plots
    plt.figure(figsize=(6,3))  # Reduced size
    sns.countplot(x=col, hue='status', data=data)
    plt.title(f"Count Plot for {col} by Status", fontsize=12)
    plt.xlabel(col.capitalize(), fontsize=10)
    plt.ylabel("Count", fontsize=10)
    plt.legend(title='Status', fontsize=8)  # Adjust legend font size
    plt.xticks(fontsize=8)
    plt.yticks(fontsize=8)
    plt.show()

    # Create stacked bar plots with percentages
    if col != 'Attrition':
        crosstab = pd.crosstab(data[col], data['status'], normalize='index')*100
        ax = crosstab.plot(kind='bar', figsize=(8, 4), stacked=True)
        plt.ylabel('Percentage Status %')
        plt.title(f"Stacked Bar Plot for {col} by Status", fontsize=12)
        plt.xlabel(col.capitalize(), fontsize=10)
        plt.xticks(fontsize=8)
        plt.yticks(fontsize=8)
        plt.legend(title='Status', fontsize=8)

        # Annotate bars with percentages
        for p in ax.patches:
            width, height = p.get_width(), p.get_height()
            x, y = p.get_xy()
            ax.annotate(f'{height:.1f}%', (x + width/2, y + height/2), ha='center', va='center', fontsize=8, color='black')

        plt.show()

# List of numerical columns to loop through
columns_to_plot = ['age', 'website_visits', 'time_spent_on_website', 'page_views_per_visit']

# Function to create both boxplots and a pair plot
def create_visualizations(data, columns, x_col):
    # Loop to create boxplots for each numerical column
    for col in columns:
        plt.figure(figsize=(10, 6))

        # Create the boxplot
        sns.boxplot(data=data, x=x_col, y=col)

        # Add labels and title
        plt.title(f'Boxplot of {col} by {x_col}')
        plt.xlabel(x_col.capitalize())
        plt.ylabel(col.capitalize())

        # Show plot
        plt.show()

    # Create a pair plot for all numerical columns with the target variable as hue
    sns.pairplot(data, hue=x_col, vars=columns, height=2.5)
    plt.suptitle("Pair Plots for Numerical Features by Status", y=1.02, fontsize=14)  # Adding a title
    plt.show()

# Call the function
create_visualizations(data, columns_to_plot, x_col="status")

# Log Transformation to address skewness above
for col in ['website_visits', 'time_spent_on_website', 'page_views_per_visit']:
    data[col] = np.log1p(data[col])  # log(1+x) to avoid log(0)

# Creating new features
data["engagement_score"] = data["time_spent_on_website"] * data["page_views_per_visit"]
data["interaction_ratio"] = data["website_visits"] / (data["page_views_per_visit"] + 1)  # Prevent division by zero

# Review data after transformation:
# Extend the list of numerical columns to include the new features
columns_to_plot = [
    'age',
    'website_visits',
    'time_spent_on_website',
    'page_views_per_visit',
    'engagement_score',
    'interaction_ratio'
]

# Function to create both boxplots and a pair plot to see if skewness is reduced
def create_visualizations(data, columns, x_col):
    # Loop to create boxplots for each numerical column
    for col in columns:
        plt.figure(figsize=(10, 6))

        # Create the boxplot
        sns.boxplot(data=data, x=x_col, y=col)

        # Add labels and title
        plt.title(f'Boxplot of {col} by {x_col}')
        plt.xlabel(x_col.capitalize())
        plt.ylabel(col.capitalize())

        # Show plot
        plt.show()

    # Create a pair plot for all numerical columns with the target variable as hue
    sns.pairplot(data, hue=x_col, vars=columns, height=2.5)
    plt.suptitle("Pair Plots for Numerical Features by Status", y=1.02, fontsize=14)  # Adding a title
    plt.show()

# Call the function
create_visualizations(data, columns_to_plot, x_col="status")

#Pairwise correlations between all the variables.
plt.figure(figsize=(10, 7))

# Select only the numeric columns
datanumbers = data.select_dtypes(include='number')

# Plot the heatmap
sns.heatmap(datanumbers.corr(), annot=True, fmt=".2f")

plt.show()

# Separating the target variable and other variables
X = data.drop(columns = 'status')
Y = data['status']

# Creating dummy variables
X = pd.get_dummies(X, drop_first = True)

# Splitting the data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 0.30, random_state = 1)

# This excludes the new features engagement score and interaction ratio and uses original features for cleaner interpretation and to avoid multicollinearity
# Function to evaluate metrics and plot confusion matrix
def metrics_score(actual, predicted, title="Confusion Matrix"):
    print(classification_report(actual, predicted))
    cm = confusion_matrix(actual, predicted)
    plt.figure(figsize=(8, 5))
    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=['Non-Converter', 'Converter'], yticklabels=['Non-Converter', 'Converter'])
    plt.title(title)
    plt.ylabel('Actual')
    plt.xlabel('Predicted')
    plt.show()

# Define features and target
X = data[['age', 'current_occupation', 'first_interaction', 'profile_completed',
          'website_visits', 'time_spent_on_website', 'page_views_per_visit',
          'last_activity', 'print_media_type1', 'print_media_type2',
          'digital_media', 'educational_channels', 'referral']]
y = data['status']

# Split the dataset into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Identify numerical and categorical columns
numerical_columns = ['age', 'website_visits', 'time_spent_on_website', 'page_views_per_visit']
categorical_columns = ['current_occupation', 'first_interaction', 'profile_completed',
                       'last_activity', 'print_media_type1', 'print_media_type2',
                       'digital_media', 'educational_channels', 'referral']

# Preprocessor: Scale numerical columns and encode categorical columns
preprocessor = ColumnTransformer(
    transformers=[
        ('num', StandardScaler(), numerical_columns),
        ('cat', OneHotEncoder(handle_unknown='ignore'), categorical_columns)
    ]
)

# Create a pipeline with preprocessing and Logistic Regression
pipeline = Pipeline(steps=[
    ('preprocessor', preprocessor),
    ('model', LogisticRegression())
])

# Train the model using the pipeline
pipeline.fit(X_train, y_train)

# Make predictions on the test set
y_pred = pipeline.predict(X_test)

# Make predictions on the training set and evaluate
y_pred_train = pipeline.predict(X_train)
print("\n--- Regression: Training Set Performance ---")
metrics_score(y_train, y_pred_train, title="Confusion Matrix (Training Set)")

# Checking performance on the testing data
y_pred_test = pipeline.predict(X_test)
print("\n--- Regression: Test Set Performance ---")
metrics_score(y_test, y_pred_test)

# Extract Coefficients and Feature Importance
# Get feature names (numerical + one-hot encoded)
feature_names = preprocessor.named_transformers_['cat'].get_feature_names_out(categorical_columns)
all_feature_names = numerical_columns + list(feature_names)

# Check for Multicollinearity using VIF
from statsmodels.stats.outliers_influence import variance_inflation_factor
import pandas as pd

# Preprocessed data for VIF calculation
X_preprocessed = pipeline.named_steps['preprocessor'].transform(X_train)
vif_data = pd.DataFrame()
vif_data["Feature"] = all_feature_names
vif_data["VIF"] = [variance_inflation_factor(X_preprocessed, i) for i in range(X_preprocessed.shape[1])]

print("\n--- Variance Inflation Factor (VIF) ---")
print(vif_data)

# Extract coefficients from the logistic regression model
coefficients = pipeline.named_steps['model'].coef_[0]

# Combine feature names, coefficients, and odds ratios
feature_importance = pd.DataFrame({
    'Feature': all_feature_names,
    'Coefficient': coefficients,
    'Odds Ratio': np.exp(coefficients)  # Convert coefficients to odds ratios
}).sort_values(by='Coefficient', ascending=False)

print("\n--- Feature Importance ---")
print(feature_importance)

# Plot the ROC Curve and AUC
y_prob = pipeline.predict_proba(X_test)[:, 1]
fpr, tpr, thresholds = roc_curve(y_test, y_prob)
plt.plot(fpr, tpr, label=f"AUC: {roc_auc_score(y_test, y_prob):.2f}")
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("ROC Curve")
plt.legend()
plt.show()

--- Regression: Training Set Performance ---
              precision    recall  f1-score   support

           0       0.86      0.92      0.89      2258
           1       0.77      0.65      0.71       970

    accuracy                           0.84      3228
   macro avg       0.82      0.78      0.80      3228
weighted avg       0.83      0.84      0.83      3228

--- Regression: Test Set Performance ---
              precision    recall  f1-score   support

           0       0.86      0.91      0.89       977
           1       0.76      0.65      0.70       407

    accuracy                           0.84      1384
   macro avg       0.81      0.78      0.79      1384
weighted avg       0.83      0.84      0.83      1384

--- Variance Inflation Factor (VIF) ---
                            Feature       VIF
0                               age  1.987195
1                    website_visits  1.163796
2             time_spent_on_website  1.240981
3              page_views_per_visit  1.144655
4   current_occupation_Professional       inf
5        current_occupation_Student       inf
6     current_occupation_Unemployed       inf
7      first_interaction_Mobile App       inf
8         first_interaction_Website       inf
9            profile_completed_High       inf
10            profile_completed_Low       inf
11         profile_completed_Medium       inf
12     last_activity_Email Activity       inf
13     last_activity_Phone Activity       inf
14   last_activity_Website Activity       inf
15             print_media_type1_No       inf
16            print_media_type1_Yes       inf
17             print_media_type2_No       inf
18            print_media_type2_Yes       inf
19                 digital_media_No       inf
20                digital_media_Yes       inf
21          educational_channels_No       inf
22         educational_channels_Yes       inf
23                      referral_No       inf
24                     referral_Yes       inf

--- Feature Importance ---
                            Feature  Coefficient  Odds Ratio
9            profile_completed_High     1.277347    3.587110
8         first_interaction_Website     1.122190    3.071574
2             time_spent_on_website     1.009186    2.743366
4   current_occupation_Professional     0.661080    1.936883
24                     referral_Yes     0.587816    1.800053
14   last_activity_Website Activity     0.381058    1.463832
0                               age     0.123472    1.131419
6     current_occupation_Unemployed     0.091862    1.096214
12     last_activity_Email Activity     0.010594    1.010651
18            print_media_type2_Yes    -0.034462    0.966125
16            print_media_type1_Yes    -0.057155    0.944448
20                digital_media_Yes    -0.082924    0.920421
22         educational_channels_Yes    -0.132644    0.875777
21          educational_channels_No    -0.162321    0.850168
1                    website_visits    -0.169436    0.844141
3              page_views_per_visit    -0.183259    0.832552
19                 digital_media_No    -0.212041    0.808932
15             print_media_type1_No    -0.237810    0.788353
17             print_media_type2_No    -0.260503    0.770664
11         profile_completed_Medium    -0.314816    0.729923
13     last_activity_Phone Activity    -0.686617    0.503276
23                      referral_No    -0.882781    0.413631
5        current_occupation_Student    -1.047907    0.350671
10            profile_completed_Low    -1.257495    0.284365
7      first_interaction_Mobile App    -1.417155    0.242403

# Define a function to plot the confusion matrix and classification report
def metrics_score(actual, predicted):
    print(classification_report(actual, predicted))
    cm = confusion_matrix(actual, predicted)
    plt.figure(figsize=(8, 5))
    sns.heatmap(cm, annot=True, fmt=".2f", xticklabels=["Converted", "Not Converted"], yticklabels=["Converted", "Not Converted"])
    plt.ylabel("Actual")
    plt.xlabel("Predicted")
    plt.title("Confusion Matrix")
    plt.show()

# Add the additional features
X = data[['age', 'current_occupation', 'first_interaction', 'profile_completed',
          'website_visits', 'time_spent_on_website', 'page_views_per_visit',
          'engagement_score', 'interaction_ratio',
          'last_activity', 'print_media_type1', 'print_media_type2',
          'digital_media', 'educational_channels', 'referral']]

#  Convert categorical variables to dummy variables - was getting error so re-getting dummy variable
X = pd.get_dummies(X, drop_first=False)  # Apply one-hot encoding with drop_first=True to avoid multicollinearity

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=1)

# Fitting the decision tree classifier on the training data
d_tree =  DecisionTreeClassifier(random_state = 7)

d_tree.fit(X_train, y_train)

DecisionTreeClassifier(random_state=7)

DecisionTreeClassifier(random_state=7)

# Checking performance on the training data
y_pred_traind1 = d_tree.predict(X_train)
print("Decision Tree: Training Data Performance:")
metrics_score(y_train, y_pred_traind1)

# Checking performance on the testing data
y_pred_test1d1 = d_tree.predict(X_test)
print("Decision Tree: Testing Data Performance:")
metrics_score(y_test, y_pred_test1d1)

Decision Tree: Training Data Performance:
              precision    recall  f1-score   support

           0       1.00      1.00      1.00      2273
           1       1.00      1.00      1.00       955

    accuracy                           1.00      3228
   macro avg       1.00      1.00      1.00      3228
weighted avg       1.00      1.00      1.00      3228

Decision Tree: Testing Data Performance:
              precision    recall  f1-score   support

           0       0.85      0.86      0.86       962
           1       0.68      0.67      0.67       422

    accuracy                           0.80      1384
   macro avg       0.76      0.76      0.76      1384
weighted avg       0.80      0.80      0.80      1384

d_tree_tuned = DecisionTreeClassifier(random_state = 7, class_weight = 'balanced')

# Grid of parameters to choose from
parameters = {'max_depth': np.arange(2, 10),
              'criterion': ['gini', 'entropy'],
              'min_samples_leaf': [5, 10, 20, 25]
             }

# Type of scoring used to compare parameter combinations - f1 score for class 1
scorer = metrics.make_scorer(f1_score, pos_label = 1)

# Run the grid search
grid_obj = GridSearchCV(d_tree_tuned, parameters, scoring = scorer, cv = 5)

grid_obj = grid_obj.fit(X_train, y_train)

# Set the classifier to the best combination of parameters
d_tree_tuned = grid_obj.best_estimator_

# Fit the best algorithm to the data
d_tree_tuned.fit(X_train, y_train)

DecisionTreeClassifier(class_weight='balanced', max_depth=np.int64(5),
                       min_samples_leaf=20, random_state=7)

DecisionTreeClassifier(class_weight='balanced', max_depth=np.int64(5),
                       min_samples_leaf=20, random_state=7)

# Checking performance on the training data
y_pred_traind2 = d_tree_tuned.predict(X_train)
print("Decision Tree Tuned: Training Data Performance:")
metrics_score(y_train, y_pred_traind2)

# Checking performance on the testing data
y_pred_testd2 = d_tree_tuned.predict(X_test)
print("Decision Tree Tuned: Test Data Performance:")
metrics_score(y_test, y_pred_testd2)

Decision Tree Tuned: Training Data Performance:
              precision    recall  f1-score   support

           0       0.94      0.84      0.89      2273
           1       0.70      0.86      0.77       955

    accuracy                           0.85      3228
   macro avg       0.82      0.85      0.83      3228
weighted avg       0.87      0.85      0.85      3228

Decision Tree Tuned: Test Data Performance:
              precision    recall  f1-score   support

           0       0.92      0.85      0.89       962
           1       0.71      0.84      0.77       422

    accuracy                           0.85      1384
   macro avg       0.82      0.85      0.83      1384
weighted avg       0.86      0.85      0.85      1384

#Reduce depth to 3 for visualization
tree_model = DecisionTreeClassifier(class_weight = 'balanced', max_depth = 3,
                       min_samples_leaf = 5, random_state = 7)

# Fit the best algorithm to the data
tree_model.fit(X_train, y_train)

DecisionTreeClassifier(class_weight='balanced', max_depth=3, min_samples_leaf=5,
                       random_state=7)

DecisionTreeClassifier(class_weight='balanced', max_depth=3, min_samples_leaf=5,
                       random_state=7)

#Plot Tree
features = list(X.columns)

plt.figure(figsize = (20, 20))

tree.plot_tree(tree_model, feature_names = features, filled = True, fontsize = 9, node_ids = True, class_names = None)

plt.show()

# Importance of features in the tree building

print (pd.DataFrame(d_tree_tuned.feature_importances_, columns = ["Imp"], index = X_train.columns).sort_values(by = 'Imp', ascending = False))

# Plotting the feature importance
importances = d_tree_tuned.feature_importances_

indices = np.argsort(importances)

plt.figure(figsize = (10, 10))

plt.title('Feature Importances')

plt.barh(range(len(indices)), importances[indices], color = 'violet', align = 'center')

plt.yticks(range(len(indices)), [features[i] for i in indices])

plt.xlabel('Relative Importance')

plt.show()

                                      Imp
first_interaction_Mobile App     0.319966
time_spent_on_website            0.259234
profile_completed_High           0.245264
current_occupation_Professional  0.052392
current_occupation_Student       0.034057
last_activity_Phone Activity     0.028137
last_activity_Website Activity   0.025097
age                              0.014836
profile_completed_Medium         0.008904
engagement_score                 0.008230
website_visits                   0.003880
first_interaction_Website        0.000000
current_occupation_Unemployed    0.000000
page_views_per_visit             0.000000
interaction_ratio                0.000000
last_activity_Email Activity     0.000000
profile_completed_Low            0.000000
print_media_type1_No             0.000000
print_media_type1_Yes            0.000000
print_media_type2_No             0.000000
print_media_type2_Yes            0.000000
digital_media_No                 0.000000
digital_media_Yes                0.000000
educational_channels_No          0.000000
educational_channels_Yes         0.000000
referral_No                      0.000000
referral_Yes                     0.000000

# Fitting the random forest tree classifier on the training data
rf_estimator = RandomForestClassifier(random_state = 7, criterion = "entropy")

rf_estimator.fit(X_train,y_train)

RandomForestClassifier(criterion='entropy', random_state=7)

RandomForestClassifier(criterion='entropy', random_state=7)

# Checking performance on the training data
y_pred_trainrf1 = rf_estimator.predict(X_train)
print("Random Forest Untuned: Training Data Performance:")
metrics_score(y_train, y_pred_trainrf1)

# Checking performance on the testing data
y_pred_testrf1 = rf_estimator.predict(X_test)
print("Random Forest Untuned: Test Data Performance:")
metrics_score(y_test, y_pred_testrf1)

Random Forest Untuned: Training Data Performance:
              precision    recall  f1-score   support

           0       1.00      1.00      1.00      2273
           1       1.00      1.00      1.00       955

    accuracy                           1.00      3228
   macro avg       1.00      1.00      1.00      3228
weighted avg       1.00      1.00      1.00      3228

Random Forest Untuned: Test Data Performance:
              precision    recall  f1-score   support

           0       0.87      0.92      0.89       962
           1       0.79      0.69      0.74       422

    accuracy                           0.85      1384
   macro avg       0.83      0.81      0.82      1384
weighted avg       0.85      0.85      0.85      1384

#Random Forest Hyperparameter Tuning
# Choose the type of classifier
rf_estimator_tuned = RandomForestClassifier(criterion = "entropy", random_state = 7)

# Grid of parameters to choose from
parameters = {"n_estimators": [110, 120],
    "max_depth": [6, 7],
    "min_samples_leaf": [20, 25],
    "max_features": [0.8, 0.9],
    "max_samples": [0.9, 1],
    "class_weight" : ["balanced",{0: 0.3, 1: 0.7}]
             }
# Type of scoring used to compare parameter combinations - f1 score for class 1
scorer = metrics.make_scorer(f1_score, pos_label = 1)

# Run the grid search
grid_obj = GridSearchCV(rf_estimator_tuned, parameters, scoring = scorer, cv = 5)

grid_obj = grid_obj.fit(X_train, y_train)

# Set the classifier to the best combination of parameters
rf_estimator_tuned_base = grid_obj.best_estimator_

# Fitting the best algorithm to the training data
rf_estimator_tuned_base.fit(X_train, y_train)

RandomForestClassifier(class_weight={0: 0.3, 1: 0.7}, criterion='entropy',
                       max_depth=7, max_features=0.8, max_samples=0.9,
                       min_samples_leaf=20, n_estimators=110, random_state=7)

RandomForestClassifier(class_weight={0: 0.3, 1: 0.7}, criterion='entropy',
                       max_depth=7, max_features=0.8, max_samples=0.9,
                       min_samples_leaf=20, n_estimators=110, random_state=7)

# Checking performance on the training data
y_pred_trainrf2 = rf_estimator_tuned_base.predict(X_train)
print("Random Forest Tuned: Training Data Performance:")
metrics_score(y_train, y_pred_trainrf2)

# Checking performance on the testing data
yF_pred_testrf2 = rf_estimator_tuned_base.predict(X_test)
print("Random Forest Tuned: Test Data Performance:")
metrics_score(y_test, yF_pred_testrf2)

Random Forest Tuned: Training Data Performance:
              precision    recall  f1-score   support

           0       0.94      0.86      0.90      2273
           1       0.72      0.86      0.78       955

    accuracy                           0.86      3228
   macro avg       0.83      0.86      0.84      3228
weighted avg       0.87      0.86      0.86      3228

Random Forest Tuned: Test Data Performance:
              precision    recall  f1-score   support

           0       0.92      0.87      0.89       962
           1       0.73      0.83      0.78       422

    accuracy                           0.85      1384
   macro avg       0.82      0.85      0.83      1384
weighted avg       0.86      0.85      0.86      1384

importances = rf_estimator_tuned_base.feature_importances_

indices = np.argsort(importances)

feature_names = list(X.columns)

plt.figure(figsize = (12, 12))

plt.title('Feature Importances')

plt.barh(range(len(indices)), importances[indices], color = 'violet', align = 'center')

plt.yticks(range(len(indices)), [feature_names[i] for i in indices])

plt.xlabel('Relative Importance')

plt.show()

from xgboost import XGBClassifier

# Choose the type of classifier
xgb_estimator_tuned = XGBClassifier(eval_metric='logloss', random_state=7)

# Grid of parameters to choose from
parameters = {
    "n_estimators": [110, 120],
    "max_depth": [6, 7],
    "learning_rate": [0.1, 0.2],
    "subsample": [0.8, 0.9],
    "colsample_bytree": [0.8, 0.9],
    "scale_pos_weight": [1, 3],  # Adjust for class imbalance
}

# Type of scoring used to compare parameter combinations - f1 score for class 1
scorer = metrics.make_scorer(f1_score, pos_label=1)

# Run the grid search
grid_obj = GridSearchCV(xgb_estimator_tuned, parameters, scoring=scorer, cv=5)

grid_obj = grid_obj.fit(X_train, y_train)

# Set the classifier to the best combination of parameters
xgb_estimator_tuned = grid_obj.best_estimator_

# Predictions and Metrics for XGBoost
y_pred_train_xgb = xgb_estimator_tuned.predict(X_train)
y_pred_test_xgb = xgb_estimator_tuned.predict(X_test)

print("XGBoost Untuned - Training Results:")
metrics_score(y_train, y_pred_train_xgb)
print("XGBoost Untuned - Test Results:")
metrics_score(y_test, y_pred_test_xgb)

XGBoost Untuned - Training Results:
              precision    recall  f1-score   support

           0       0.97      0.98      0.97      2273
           1       0.94      0.92      0.93       955

    accuracy                           0.96      3228
   macro avg       0.95      0.95      0.95      3228
weighted avg       0.96      0.96      0.96      3228

XGBoost Untuned - Test Results:
              precision    recall  f1-score   support

           0       0.88      0.92      0.90       962
           1       0.79      0.71      0.75       422

    accuracy                           0.86      1384
   macro avg       0.84      0.81      0.82      1384
weighted avg       0.85      0.86      0.85      1384

# Choose the type of classifier
xgb_estimator_tuned = XGBClassifier(eval_metric='logloss', random_state=7)

# Grid of parameters to choose from
parameters = {
    "n_estimators": [100, 200],
    "max_depth": [3, 5, 7],
    "learning_rate": [0.01, 0.1, 0.2],
    "subsample": [0.8, 0.9, 1.0],
    "colsample_bytree": [0.8, 0.9, 1.0],
    "scale_pos_weight": [1, 3],  # Adjust for class imbalance
    "reg_alpha": [0, 0.1, 1],  # L1 regularization
    "reg_lambda": [1, 1.5, 2]  # L2 regularization
}

# Type of scoring used to compare parameter combinations - f1 score for class 1
scorer = metrics.make_scorer(f1_score, pos_label=1)

# Run the randomized search
random_search = RandomizedSearchCV(xgb_estimator_tuned, parameters, scoring=scorer, cv=5, n_iter=50, n_jobs=multiprocessing.cpu_count())
random_search = random_search.fit(X_train, y_train)

# Set the classifier to the best combination of parameters
xgb_estimator_tuned = random_search.best_estimator_

# Predictions and Metrics for XGBoost
y_pred_train_xgb2 = xgb_estimator_tuned.predict(X_train)
y_pred_test_xgb2 = xgb_estimator_tuned.predict(X_test)

print("XGBoost Tuned - Training Results:")
metrics_score(y_train, y_pred_train_xgb2)
print("XGBoost Tuned - Test Results:")
metrics_score(y_test, y_pred_test_xgb2)

XGBoost Tuned - Training Results:
              precision    recall  f1-score   support

           0       0.90      0.95      0.92      2273
           1       0.85      0.74      0.79       955

    accuracy                           0.89      3228
   macro avg       0.88      0.84      0.86      3228
weighted avg       0.88      0.89      0.88      3228

XGBoost Tuned - Test Results:
              precision    recall  f1-score   support

           0       0.87      0.94      0.90       962
           1       0.83      0.69      0.76       422

    accuracy                           0.86      1384
   macro avg       0.85      0.82      0.83      1384
weighted avg       0.86      0.86      0.86      1384

# SHAP values for feature importance
explainer = shap.Explainer(xgb_estimator_tuned)
shap_values = explainer(X_train)

# Plot the SHAP summary plot for feature importance
shap.summary_plot(shap_values, X_train)

# Compute metrics for each model
metrics_dict = {
    'Model': ['Decision Tree (Tuned)', 'Random Forest (Tuned)', 'XGBoost (Tuned)'],
    'Weighted F1 Score': [
        f1_score(y_test, d_tree_tuned.predict(X_test), average='weighted'),
        f1_score(y_test, rf_estimator_tuned_base.predict(X_test), average='weighted'),
        f1_score(y_test, xgb_estimator_tuned.predict(X_test), average='weighted')
    ],
    'Class 0 F1 Score': [
        f1_score(y_test, d_tree_tuned.predict(X_test), labels=[0], average='macro'),
        f1_score(y_test, rf_estimator_tuned_base.predict(X_test), labels=[0], average='macro'),
        f1_score(y_test, xgb_estimator_tuned.predict(X_test), labels=[0], average='macro')
    ],
    'Class 1 F1 Score': [
        f1_score(y_test, d_tree_tuned.predict(X_test), labels=[1], average='macro'),
        f1_score(y_test, rf_estimator_tuned_base.predict(X_test), labels=[1], average='macro'),
        f1_score(y_test, xgb_estimator_tuned.predict(X_test), labels=[1], average='macro')
    ],
    'Accuracy': [
        accuracy_score(y_test, d_tree_tuned.predict(X_test)),
        accuracy_score(y_test, rf_estimator_tuned_base.predict(X_test)),
        accuracy_score(y_test, xgb_estimator_tuned.predict(X_test))
    ]
}

# Create a DataFrame to display results as a table
metrics_table = pd.DataFrame(metrics_dict)

# Display the table
print("\n--- Metrics Comparison Table ---")
print(metrics_table)

--- Metrics Comparison Table ---
                   Model  Weighted F1 Score  Class 0 F1 Score  \
0  Decision Tree (Tuned)           0.851347          0.886486   
1  Random Forest (Tuned)           0.856399          0.891863   
2        XGBoost (Tuned)           0.859045          0.904618   

   Class 1 F1 Score  Accuracy  
0          0.771242  0.848266  
1          0.775556  0.854046  
2          0.755155  0.862717

	count	mean	std	min	25%	50%	75%	max
age	4612.0	46.201214	13.161454	18.0	36.00000	51.000	57.00000	63.000
website_visits	4612.0	3.566782	2.829134	0.0	2.00000	3.000	5.00000	30.000
time_spent_on_website	4612.0	724.011275	743.828683	0.0	148.75000	376.000	1336.75000	2537.000
page_views_per_visit	4612.0	3.026126	1.968125	0.0	2.07775	2.792	3.75625	18.434
status	4612.0	0.298569	0.457680	0.0	0.00000	0.000	1.00000	1.000

ExtraaLearn Project¶

Context¶

Objective¶

Data Description¶

Importing necessary libraries and data¶

Data Overview¶

Exploratory Data Analysis (EDA)¶

Data Preprocessing¶

Preparing the data for modeling¶

Building a Decision Tree model¶

Model Performance evaluation and improvement¶

Decision Tree - Hyperparameter Tuning¶

Building a Random Forest model¶

Model Performance evaluation and improvement¶

Actionable Insights and Recommendations¶

XGBoost Regressor (Not in Notebook, But I would like to check)¶

Final Takeaways¶

	ID	age	current_occupation	first_interaction	profile_completed	website_visits	time_spent_on_website	page_views_per_visit	last_activity	print_media_type1	print_media_type2	digital_media	educational_channels	referral	status
0	EXT001	57	Unemployed	Website	High	7	1639	1.861	Website Activity	Yes	No	Yes	No	No	1
1	EXT002	56	Professional	Mobile App	Medium	2	83	0.320	Website Activity	No	No	No	Yes	No	0
2	EXT003	52	Professional	Website	Medium	3	330	0.074	Website Activity	No	No	Yes	No	No	0
3	EXT004	53	Unemployed	Website	High	4	464	2.057	Website Activity	No	No	No	No	No	1
4	EXT005	23	Student	Website	High	4	600	16.914	Email Activity	No	No	No	No	No	0

	ID	age	current_occupation	first_interaction	profile_completed	website_visits	time_spent_on_website	page_views_per_visit	last_activity	print_media_type1	print_media_type2	digital_media	educational_channels	referral	status
4607	EXT4608	35	Unemployed	Mobile App	Medium	15	360	2.170	Phone Activity	No	No	No	Yes	No	0
4608	EXT4609	55	Professional	Mobile App	Medium	8	2327	5.393	Email Activity	No	No	No	No	No	0
4609	EXT4610	58	Professional	Website	High	2	212	2.692	Email Activity	No	No	No	No	No	1
4610	EXT4611	57	Professional	Mobile App	Medium	1	154	3.879	Website Activity	Yes	No	No	No	No	0
4611	EXT4612	55	Professional	Website	Medium	4	2290	2.075	Phone Activity	No	No	No	No	No	0